package com.etoak.crawl.main;

import com.etoak.crawl.link.LinkFilter;
import com.etoak.crawl.link.Links;
import com.etoak.crawl.page.AgencyIp;
import com.etoak.crawl.page.Page;
import com.etoak.crawl.page.PageParserTool;
import com.etoak.crawl.page.RequestAndResponseTool;
import com.etoak.crawl.pojo.Product;
import com.etoak.crawl.pojo.Twotitle;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;


public class MyCrawlerlist {

    /**
     * 使用种子初始化 URL 队列
     *
     * @param seeds 种子 URL
     * @return
     */
    private void initCrawlerWithSeeds(String[] seeds) {
        for (int i = 0; i < seeds.length; i++) {
            Links.addUnvisitedUrlQueue(seeds[i]);
        }
    }


    /**
     * 抓取过程
     *
     * @param seeds
     * @return
     */
    public List<Product> crawling(String[] seeds) throws IOException {
        List<Product> prolist = new ArrayList<Product>();
        //初始化 URL 队列
        initCrawlerWithSeeds(seeds);

        //定义过滤器，提取以 http://www.baidu.com 开头的链接
        LinkFilter filter = new LinkFilter() {
            public boolean accept(String url) {
                if (url.startsWith("http://www.bzjw.com"))
                    return true;
                else
                    return false;
            }
        };

        //循环条件：待抓取的链接不空且抓取的网页不多于 1000
        while (!Links.unVisitedUrlQueueIsEmpty() && Links.getVisitedUrlNum() <= 1000) {

            //先从待访问的序列中取出第一个；
            String visitUrl = (String) Links.removeHeadOfUnVisitedUrlQueue();
            if (visitUrl == null) {
                continue;
            }

            //根据URL得到page;
            Page page = RequestAndResponseTool.sendRequstAndGetResponse(visitUrl);

            //对page进行处理： 访问DOM的某个标签 --- 二级分类查询
            Elements es = PageParserTool.select(page, "ul#list_gb");


            //解析二级分类名称和链接
            List<Twotitle> ttlist = new ArrayList<Twotitle>();

            Elements itemsench = es.select("a");
            int i =1;
            for (Element element : itemsench) {
                Twotitle tt = new Twotitle();
                String linkhref = element.attr("href");
                String linktext = element.text();

                //链接
                linkhref = "http://www.bzjw.com"+linkhref;
                //截取分类标题
                linktext = linktext.substring(0, linktext.length() - 2) + linktext.substring(linktext.length()-2, linktext.length()).replaceAll("\\d+","");
                //System.err.println("链接:"+linkhref+"=====标题:"+linktext);

                tt.setId(i);
                i++;
                tt.setTitle(linktext);
                tt.setUrl(linkhref);
                tt.setOneid(1);
                ttlist.add(tt);
            }


            //二级分类
            for (Twotitle twotitle : ttlist) {
                prolist.addAll(crawlings(new String[]{twotitle.getUrl()},twotitle.getTitle()))    ;

            }

            //crawling(new String[]{"http://www.bzjw.com/standard/index.asp?class_id=526"});

        }
        return prolist;
    }


    /**
     * 抓取产品列表
     * @param seeds
     * @throws IOException
     */
    public List<Product> crawlings(String[] seeds,String title) throws IOException {

        List<Product> prolist = new ArrayList<Product>();
        MyCrawler crawler = new MyCrawler();
        //初始化 URL 队列
        initCrawlerWithSeeds(seeds);

        //定义过滤器，提取以 http://www.baidu.com 开头的链接
        LinkFilter filter = new LinkFilter() {
            public boolean accept(String url) {
                if (url.startsWith("http://www.bzjw.com"))
                    return true;
                else
                    return false;
            }
        };

        //循环条件：待抓取的链接不空且抓取的网页不多于 1000
        while (!Links.unVisitedUrlQueueIsEmpty() && Links.getVisitedUrlNum() <= 1000) {

            //先从待访问的序列中取出第一个；
            String visitUrl = (String) Links.removeHeadOfUnVisitedUrlQueue();
            if (visitUrl == null) {
                continue;
            }

            //根据URL得到page;
            Page page = RequestAndResponseTool.sendRequstAndGetResponse(visitUrl);

            //对page进行处理： 访问DOM的某个标签 --- 产品查询
            Elements es = PageParserTool.select(page, "ul#list_gb");

            Elements links = es.select("a[target]");
            for (Element link : links) {
                String linkHref = link.attr("href");
                linkHref = "http://www.bzjw.com"+linkHref;
                System.err.println("产品路径:"+linkHref);
                //拉取产品
              prolist.addAll(crawler.crawlingpro(new String[]{linkHref},title))   ;
            }
            System.out.println("------------------------------------------");

            //crawling(new String[]{"http://www.bzjw.com/standard/index.asp?class_id=526"});


        }
        return prolist;
    }


    //main 方法入口
    public static void main(String[] args) throws IOException {
       MyCrawlerlist crawler = new MyCrawlerlist();
        crawler.crawling(new String[]{"http://www.bzjw.com/standard/index.asp?class_id=526"});
    }
}
